package org.example.httpWeb;

import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * 并行和串行的方式爬取10个网页上的data数据，并且相加求和
 */
public class httpWeb {
        static List<Integer> sum = new ArrayList<>();


        public static void main(String[] args) {

            ExecutorService executor = Executors.newFixedThreadPool(10);


            List<String> urls = new ArrayList<>();
            String prefix = "http://dy-public.oss-cn-shenzhen.aliyuncs.com/interviewTestData/";
            String suffix = ".txt";
            for (int i = 1;i<=10;i++) {
                urls.add(prefix + i + suffix);
            }

            //并行执行
            urls.parallelStream()
                    .forEach(url -> executor.submit(() -> handler(url)));

            // 等待所有任务完成（这只是一个简单的示例，实际中可能需要更复杂的同步机制）
            executor.shutdown();


            //串行执行
//            urls.stream().forEach(item-> handler(item));

            System.out.println(sum);
            System.out.println(sum.stream().reduce(Integer::sum).orElse(0));

        }

    private static void handler(String url) {
        Request request = new Request.Builder()
                .url(url) // 替换为你想要爬取的网站URL
                .build();
        OkHttpClient client = new OkHttpClient();
        try (Response response = client.newCall(request).execute()) {
            if (!response.isSuccessful()) throw new IOException("Unexpected code " + response);

            // 将响应体转换为字符串
            String responseBody = response.body().string();

            // 使用Jsoup解析HTML
            Document doc = Jsoup.parse(responseBody);

            // 使用CSS选择器提取数据
            Elements elements = doc.select("de");
            if (elements.size() <= 0) {
                elements = doc.select("def");
            }// 例如，提取所有链接
            for (Element element : elements) {
                String text = element.text(); // 获取<def>元素的文本内容
                // 使用正则表达式或字符串处理来提取"data"后面的数据
                String data = text.replaceAll(".*data : (\\d+);.*", "$1"); // 使用正则表达式提取数字

                sum.add(Integer.valueOf(data));
            }

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
